#ifndef TOKENIZER_HEADER
#define TOKENIZER_HEADER

enum TokenType {
	TOKEN_KEYWORD = 0,
	TOKEN_VARNAME,      // asdf
	TOKEN_NUMBER,       // 1234, 0x1234, 01234
	TOKEN_STRING,		// "~~~~~~"
	TOKEN_OPERATOR,		// +, -, *, <, <=, ==, >=, >, !=, =, and, or, xor
	TOKEN_LPAREN,		// (
	TOKEN_RPAREN,		// )
	TOKEN_SEPERATOR	// '
};

typedef struct _token {
	int m_tokentype;
	const char *m_begin;
	const char *m_end;    //*m_end are not included in token. begin ~ end scheme
	void* m_parsed;
}Token;

enum KeywordType {
	KEYWORD_VAR,
	KEYWORD_IF,
	KEYWORD_ELIF,
	KEYWORD_ELSE,
	KEYWORD_FOR,
	KEYWORD_WHILE,
	KEYWORD_FUNCTION,
	KEYWORD_CONDITION,
	KEYWORD_ACTION,
	KEYWORD_TRIGGER,
	KEYWORD_END
};

Token* TokenizeLine(const char* line, int *tokenn);

#endif